R Language

CRAN Packages

sparklyr 0.4 - Initial Release

library(sparklyr)                                    # R Interface to Apache Spark
spark_install()                                      # Install Apache Spark
sc <- spark_connect(master = "local")                # Connect to Spark cluster

cars_tbl <- spark_read_csv(sc, "cars", "mtcars/")    # Read data in Spark

summarize(cars_tbl, n = n())                         # Count records with dplyr
dbGetQuery(sc, "SELECT count(*) FROM cars")          # Count records with DBI

ml_linear_regression(cars_tbl, mpg ~ wt + cyl)       # Perform linear regression

spark_context(sc) %>% invoke("version")              # Extend sparklyr with Scala

sparklyr 0.5 - Connections

sc <- spark_connect(method = "databricks")           # Connect to Databricks cluster
sc <- spark_connect(master = "http://livy-server")   # Connect through Apache Livy 

sparklyr 0.6 - Distributed R

spark_apply(cars_tbl, nrow)                          # Extend sparklyr with R

sparklyr 0.7 - Pipelines

pipeline <- ml_pipeline(sc) %>%                      # Define Spark pipeline
  ft_r_formula(mpg ~ wt + cyl) %>%                   # Add formula transformation
  ml_linear_regression()                             # Add model to pipeline

fitted <- ml_fit(pipeline, cars_tbl)                 # Fit pipeline

sparklyr 0.8 - MLeap and Graphs

library(mleap)                                       # Install MLeap package
install_maven()                                      # Install Maven
install_mleap()                                      # Install MLeap

transformed <- ml_transform(fitted, mtcars_tbl)      # Fit pipeline with dataset
ml_write_bundle(fitted, transformed, "model.zip")    # Export model with MLeap

library(graphframes)

g <- gf_graphframe(edges = edges_tbl)
gf_pagerank(g, tol = 0.01)

sparklyr 0.9 - Streams

stream_read_text(sc, "s3a://your-s3-bucket/") %>%
  spark_apply(
    ~webreadr::read_s3(paste(c(.x$line, ""), collapse = "\n")),
    columns = lapply(webreadr::read_s3("\n"), class)
  ) %>%
  group_by(uri) %>%
  summarize(n = n()) %>%
  arrange(desc(n)) %>%
  stream_write_memory("urls_stream", mode = "complete")

sparklyr 1.0 - Arrow

• Arrow enables faster and larger data transfers between Spark and R.
• XGBoost enables training gradient boosting models over distributed datasets.
• Broom converts Spark’s models into tidy formats that you know and love.
• TFRecords writes TensorFlow records from Spark to support deep learning workflows.

What is Arrow?

Arrow is a cross-language development platform for in-memory data.

The Feather project

The Ursa project

The Arrow R package

Install Arrow

devtools::install_github(
  "apache/arrow",
  subdir = "r",
  ref = "apache-arrow-0.13"
)

Then use Arrow with Spark and R:

library(arrow)

Copy with Arrow

library(arrow)
copy_to(sc, data.frame(y = 1:rows), overwrite = T)

Collect with Arrow

library(arrow)
sdf_len(sc, rows) %>% collect()

Transform with Arrow

library(arrow)
sdf_len(sc, rows) %>% spark_apply(~.x/2) %>% count() %>% collect()

Resources

• Docs: spark.rstudio.com
• GitHub: github.com/rstudio/sparklyr
• Blog: blog.rstudio.com/tags/sparklyr
• R Help: community.rstudio.com
• Spark Help: stackoverflow.com/tags/sparklyr
• Issues: github.com/rstudio/sparklyr/issues
• Chat: gitter.im/rstudio.sparklyr